#!/usr/bin/env pyth0n
# -*- coding: utf-8 -*-
from pyspark import SparkConf, SparkContext

import os

from pyspark.sql import SparkSession

"""
-------------------------------------------------
   Description :	TODO：
   SourceFile  :	pyspark
   Author      :	KunLin Yang
   Date	       :	2024/2/28
-------------------------------------------------
"""

if __name__ == '__main__':
    # todo:0-设置系统环境变量
    os.environ['JAVA_HOME'] = '/export/server/jdk1.8.0_241'
    os.environ['SPARK_HOME'] = '/export/server/spark'
    os.environ['PYSPARK_PYTHON'] = '/root/anaconda3/bin/python3'
    os.environ['PYSPARK_DRIVER_PYTHON'] = '/root/anaconda3/bin/python3'

    # todo:1-构建SparkContext
    spark = SparkSession.builder \
        .appName('app name') \
        .master('local[*]') \
        .getOrCreate()
    sc = spark.sparkContext

    # todo:2-数据处理：读取、转换、保存
    # step1: 读取数据
    # step2: 处理数据
    # step3: 保存结果

    # todo:3-关闭SparkContext
    sc.stop()
